# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from government.items import GovernmentItem
from government.db.dbhelper import engine, ArticleModel

from datetime import datetime
import copy


class GovernmentPipeline(object):

    def open_spider(self, spider):
        self.items = []

    def process_item(self, item, spider):
        if isinstance(item, GovernmentItem):
            if item.get('publish_date'):
                item['publish_date'] = datetime.strptime(item['publish_date'], "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S")
            if item.get('expired_date'):
                item['expired_date'] = datetime.strptime(item['expired_date'], "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S")
            self.items.append(copy.deepcopy(item))

    def close_spider(self, spider):
        if len(self.items) > 0:
            conn = engine.connect()
            try:
                conn.execute(ArticleModel.__table__.insert(), self.items)
            except Exception as e:
                print('插入数据出错， 错误信息：%s' % e)
            finally:
                conn.close()
# TODO:redis数据库，用于去重检查